<?php

require 'lib/curl.php';
require 'lib/dom.php';
include 'lib/filter.php';

date_default_timezone_set('Europe/Warsaw');
define('BASE_PATH', realpath(__DIR__));
error_reporting(E_ALL);
ini_set('display_errors', 1);

$options = getopt("u:o:f:");

$stdout = fopen('php://stdout', 'w');

$level = 1;
$maxRecursive = 1;
$maxRedirect = 2;
$redirectCount = 0;

if (!isset($options['u'])) {
    echo('Błąd# Musisz podać adres URL (-u "url")' . "\r");
    return;
}

$force      = isset($options['f']) ? $options['f'] : false;

$errors     = array();

$dbName     = "websearchboot";
$dbUser     = "root";
$dbPwd      = "";
$dbHost     = "localhost";

$mysqli = new mysqli($dbHost, $dbUser, $dbPwd, $dbName);
if ($mysqli->connect_errno) {
    echo "Failed to connect to MySQL: (" . $mysqli->connect_errno . ") " . $mysqli->connect_error;
}

$tmpdbname = time();

if ($mysqli->query("CREATE TEMPORARY TABLE tmp" . $tmpdbname . " (
    id INT NOT NULL AUTO_INCREMENT,     
    urlHash VARCHAR(255) NOT NULL,
    url VARCHAR(255) NOT NULL,
    title VARCHAR(255) NULL,
    text VARCHAR(255) NULL,
    rel VARCHAR(255) NULL,
    urlOriginal VARCHAR(255) NOT NULL,
    urlFrom VARCHAR(255) NOT NULL,
    httpCode VARCHAR(255) NULL,
    contentType VARCHAR(255) NULL,
    pageTitle VARCHAR(255) NULL
    , created DATETIME ,PRIMARY KEY (id))ENGINE=MEMORY") === TRUE) {

} else {
    echo($mysqli->error);
    exit(1);
}

$startDate = time();
$endDate = 0;


st:

$ch = curl_init();

$headers = array('Accept-Language: pl,en-us;q=0.7,en;q=0.3',
    'Accept-Charset: ISO-8859-2,utf-8;q=0.7,*;q=0.7');

$curlConfig = array(
    CURLOPT_HTTPHEADER => $headers,
    CURLOPT_REFERER => 'http://websearchboot.phpdev.c0.pl/boot.html',
    CURLOPT_USERAGENT => 'Mozilla/5.0; WebSearchBoot(verision=1.0); (X11; U; Linux i686; pl; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3',
    CURLOPT_RETURNTRANSFER => true,
    CURLOPT_HEADER => 1,
    CURLOPT_CONNECTTIMEOUT => 5,
    CURLOPT_COOKIEFILE => dirname(__FILE__) . '/cookies.txt',
    CURLOPT_COOKIEJAR => dirname(__FILE__) . '/cookies.txt',
);

$url = preg_replace('/\/$/', '', $options['u']);

$runOnce = isset($options['o']) ? (bool) $options['o'] : false;

$newI = 1;
try {
    redirect:
    get:
    $urlConf = parse_url($url);

    $loadStart = time();
    $loadEnd = 0;
    $r = get($url);
    $loadEnd = time();

//printf("\nLoading: " . round(($loadEnd - $loadStart) / 60) . '.' . (($loadEnd - $loadStart) % 60) . " min");

    $info = getInfo();
    $content_type = explode(';', $info['content_type']);

//$content_type[0] == "text/html"

    if ($info['http_code'] == 200) {

        $dom = new Dom(null, 'utf-8', $r);

        $runHash = array(md5($url));

        $urlItem = false;

        $row = $mysqli->query("SELECT * FROM tmp$tmpdbname where url like '$url'");
        if ($row) {
            $urlItem = $row->fetch_object();
            if ($urlItem) {
                $mysqli->query("update tmp$tmpdbname set httpCode='{$info['http_code']}',contentType='{$content_type[0]}' where id={$urlItem->id}");
            }
        }
        
        $pagetitle = substr($dom->getElementsByTagName('title')->length > 0 ? $dom->getElementsByTagName('title')->item(0)->textContent : '', 0, 160);
        
        if ($urlItem) {
            $mysqli->query("update tmp$tmpdbname set pageTitle='{$pagetitle}' where id={$urlItem->id}");
        }

        aLink($dom, $dom->getElementsByTagName('a'));
        aLink($dom, $dom->getElementsByTagName('link'));
        aLink($dom, $dom->getElementsByTagName('script'), 'src');
        aLink($dom, findLink($r));
        
    } elseif ($info['http_code'] == (301 || 302)) {


        if ($redirectCount < $maxRedirect) {
            $redirectCount +=1;

            if (isset($info['redirect_url']) && !empty($info['redirect_url']) && preg_match('#' . str_replace('www.', '', $urlConf['host']) . '#', str_replace('www.', '', $info['redirect_url'])) == 1) {
                $url = $info['redirect_url'];

                echo "Redirect[$redirectCount] to: $url\n\n";
                goto redirect;
            }
        }
    }

    $result = $mysqli->query("SELECT id,url FROM tmp$tmpdbname WHERE url like '%{$urlConf['host']}%' order by created asc limit 1");
    $resultNoHost = $mysqli->query("SELECT count(*) as c FROM tmp$tmpdbname WHERE url not like '%{$urlConf['host']}%' order by created asc limit 1");
    $resCount = $mysqli->query("SELECT count(*) as c FROM tmp$tmpdbname");

    $count = $resCount ? $resCount->fetch_object()->c : 0;
    $countNoHost = $resultNoHost ? $resultNoHost->fetch_object()->c : 0;
    $proc = $count > 0 ? ( round(($newI / $count) * 100, 2) ) : 0;
    $proc = $newI == $count ? 100 : $proc;

    fwrite($stdout, "$newI\t$count\t$countNoHost\t" . str_pad("Completed $proc %", 32) . "\r");

    if ($newI < $count && !$runOnce) {

        $newUrl = $result->fetch_object();
        if ($newUrl) {
            $url = $newUrl->url;
            $mysqli->query("UPDATE tmp$tmpdbname SET created='" . date("Y-m-d H:i:s", time()) . "' where id=" . $newUrl->id . " ");
            $newI +=1;
            goto get;
        }
    } else {
        printf("\n" . $mysqli->error);
    }

    curl_close($ch);
    $level+=1;

    usleep((100000) * ( rand(1, 3)));

    if ($level <= $maxRecursive)
        goto st;

    $filename = $urlConf['host'] . '.txt';

    $result = null;
    $resultNoHost = null;
    $resCount = null;

    $result = $mysqli->query("SELECT * FROM tmp$tmpdbname 
    where (httpCode like '200' or httpCode like '304') AND
    LENGTH(text) > 5 AND
    (rel NOT LIKE 'noindex' and rel NOT LIKE 'nofollow') 
    AND url like '%{$urlConf['host']}%'
order by id asc");

    $resultA = $mysqli->query("SELECT * FROM tmp$tmpdbname 
order by id asc");

    $resultNoHost = $mysqli->query("SELECT * FROM tmp$tmpdbname 
    where url not like '%{$urlConf['host']}%'
order by id asc");

    $dir = preg_replace(array('#[^0-9a-z_\-]#i'), array(''), $urlConf['host']);

    if (!file_exists(realpath(__DIR__) . DIRECTORY_SEPARATOR . 'out' . DIRECTORY_SEPARATOR . $dir)) {
        mkdir(realpath(__DIR__) . DIRECTORY_SEPARATOR . 'out' . DIRECTORY_SEPARATOR . $dir, 0777);
    }

    $dir = 'out' . DIRECTORY_SEPARATOR . $dir;

    file_put_contents(realpath(__DIR__) . DIRECTORY_SEPARATOR . $dir . DIRECTORY_SEPARATOR . $filename, '');
    file_put_contents(realpath(__DIR__) . DIRECTORY_SEPARATOR . $dir . DIRECTORY_SEPARATOR . 'nohost-' . $filename, '');
    file_put_contents(realpath(__DIR__) . DIRECTORY_SEPARATOR . $dir . DIRECTORY_SEPARATOR . 'all-' . $filename, '');

    if ($result) {

        $all = $result->fetch_all();
        $allc = $mysqli->query("SELECT count(*) as c FROM tmp$tmpdbname")->fetch_object()->c;

        printf("\nIndex " . count($all) . "\tfrom $allc");

        foreach ($all as $item) {

            file_put_contents(realpath(__DIR__) . DIRECTORY_SEPARATOR . $dir . DIRECTORY_SEPARATOR . $filename, $item[1] . "\t" . $item[2] . "\t" . $item[3] . "\t" . $item[4] . "\t" . getDir($item[5]) . "\t" . $item[6] . "\t" . $item[7] . "\t" . $item[8] . "\t" . $item[9] . "\t" . $item[10] . "\t" . $item[11] . '' . PHP_EOL, FILE_APPEND);
        }
    }

    if ($resultNoHost) {

        $all = $resultNoHost->fetch_all();
        $allc = $mysqli->query("SELECT count(*) as c FROM tmp$tmpdbname")->fetch_object()->c;

        printf("\nIndex noHost " . count($all) . "\tfrom $allc");

        foreach ($all as $item) {

            file_put_contents(realpath(__DIR__) . DIRECTORY_SEPARATOR . $dir . DIRECTORY_SEPARATOR . 'nohost-' . $filename, $item[1] . "\t" . $item[2] . "\t" . $item[3] . "\t" . $item[4] . "\t" . getDir($item[5]) . "\t" . $item[6] . "\t" . $item[7] . "\t" . $item[8] . "\t" . $item[9] . "\t" . $item[10] . "\t" . $item[11] . '' . PHP_EOL, FILE_APPEND);
        }
    }

    if ($resultA) {

        $all = $resultA->fetch_all();
        $allc = $mysqli->query("SELECT count(*) as c FROM tmp$tmpdbname")->fetch_object()->c;

        printf("\nIndex all " . count($all) . "\tfrom $allc");

        foreach ($all as $item) {

            file_put_contents(realpath(__DIR__) . DIRECTORY_SEPARATOR . $dir . DIRECTORY_SEPARATOR . 'all-' . $filename, $item[1] . "\t" . $item[2] . "\t" . $item[3] . "\t" . $item[4] . "\t" . getDir($item[5]) . "\t" . $item[6] . "\t" . $item[7] . "\t" . $item[8] . "\t" . $item[9] . "\t" . $item[10] . "\t" . $item[11] . '' . PHP_EOL, FILE_APPEND);
        }
    }

    $mysqli->query("DROP TABLE tmp$tmpdbname");
    $mysqli->close();

    upload:
//s$filename = 'www.deltafan.pl.txt';
    /* upload(
      'http://websearchboot.phpdev.c0.pl/?page=uploadsitemap', array(
      'sitemap' => '@' . BASE_PATH . '/' . $filename,
      'sitemap-nohosts' => '@' . BASE_PATH . '/nohost-' . $filename,
      )
      ); */

    $endDate = time();
    $dif = $endDate - $startDate;
    printf("\n $dif sec\t" . round(($dif / 60)) . "\t" . round(($dif / 60)) . '.' . (($dif % 60))) . "\n";
    //print_r($urlConf);
} catch (Exception $e) {
    var_dump($e);
}

exit(0);
